package com.bone.spider.crawler;

import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;

import org.apache.commons.lang3.math.NumberUtils;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.bone.spider.domain.DoubanGroup;
import com.bone.spider.repository.DoubanGroupRepository;
import com.bone.spider.utils.UrlUtils;
import com.webcollector.model.CrawlDatums;
import com.webcollector.model.Page;
import com.webcollector.plugin.rocks.BreadthCrawler;

import okhttp3.Cookie;

/**
 * Crawling news from github news
 *
 * @author hu
 */
public class DoubanGroupCrawler extends BreadthCrawler {
    
    private List<String> names = Arrays.asList("上海", "黄浦", "徐汇", "长宁", "静安", "普陀", "虹口", "杨浦", "闵行", "宝山", "嘉定", "浦东");
    private DoubanGroupRepository doubanGroupRepository;
    private ConcurrentHashMap<String, List<Cookie>> cookieStore = new ConcurrentHashMap<>();

    
    /**
     * @param crawlPath crawlPath is the path of the directory which maintains
     *                  information of this crawler
     * @param autoParse if autoParse is true,BreadthCrawler will auto extract
     *                  links which match regex rules from pag
     */
    public DoubanGroupCrawler(String crawlPath, boolean autoParse, DoubanGroupRepository doubanGroupRepository) throws UnsupportedEncodingException {
        super(crawlPath, autoParse);
        this.doubanGroupRepository = doubanGroupRepository;
        setRequester(new DoubanHttpRequest(cookieStore));
        /*start pages*/
        // e.g. https://www.douban.com/group/search?cat=1019&q=浦东+租房, use urlEncode
        // 翻页 https://www.douban.com/group/search?start=20&cat=1019&sort=time&q=
        
        for (String name : names) {
            String searchKey = URLEncoder.encode(name + " 租房", "utf-8");
            this.addSeed("https://www.douban.com/group/search?cat=1019&q=" + searchKey, searchKey);
        }
        setThreads(1);
    }

    @Override
    public void visit(Page page, CrawlDatums next) {
        /*if page is news page*/
        String url = page.url(); 
        Elements results = page.select("div[class=result]");
        if (results.size() == 20) {
            Map<String, String> params = UrlUtils.getParams(url);
            String type = page.crawlDatum().type();
            int pageNumber;
            if (url.contains("start=")) {
                pageNumber = NumberUtils.toInt(params.get("start")) + 20;
            } else {
                pageNumber = 20;
            }
            next.add("https://www.douban.com/group/search?cat=1019&sort=time&q=" + type + "&start=" + pageNumber, type);
        }
        List<DoubanGroup> doubanGroupList = results.stream().map(element -> {
            Element info = element.select("a[class=nbg]").get(0);
            String href = info.attr("href");
            String id = href.substring(29, href.lastIndexOf('/'));
            String text = element.select("div[class=info]").text();
            int num = NumberUtils.toInt(text.substring(0, text.indexOf(' ')));
            Optional<DoubanGroup> doubanGroupOpt = doubanGroupRepository.findById(id);
            DoubanGroup doubanGroup;
            if (doubanGroupOpt.isPresent()) {
                doubanGroup = doubanGroupOpt.get();
                if (doubanGroup.getNum() == num) {
                    return null;
                }
                doubanGroup.setNum(num);
            } else {
                doubanGroup = new DoubanGroup();
                String title = info.attr("title");
                String pic = info.getElementsByTag("img").get(0).attr("src");
                String description = element.getElementsByTag("p").get(0).text();
                doubanGroup.setId(id);
                doubanGroup.setPic(pic);
                doubanGroup.setTitle(title);
                doubanGroup.setNum(num);
                doubanGroup.setDescription(description);
            }
            return doubanGroup;
        }).filter(Objects::nonNull).collect(Collectors.toList());
        doubanGroupRepository.saveAll(doubanGroupList);
    }

}